import facade
import parsel
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
configfile = BaseDir.get_new_path(curPath, "db.ini")


class DownJournal(object):
    def __init__(self):
        self.logger = facade.get_streamlogger()
        self.mysqlutile = facade.MysqlUtiles(configfile, "db", logger=self.logger)
        self.BaseUrl = "https://www.astm.org/DIGITAL_LIBRARY/JOURNALS/"
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
            "Accept-Encoding": "gzip, deflate",
            "Host": "www.astm.org",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
        }

    def select(self):
        sql = "select sourcename,url,journal from journal where stat=0"
        rows = self.mysqlutile.SelectFromDB(sql)
        for row in rows:
            url2 = row[1].replace("/index.html", "")
            url = self.BaseUrl + url2 + "/" + row[2].lower() + "_issues.html"
            self.logger.info(url)
            BoolResult, errString, r = facade.BaseRequest(url,
                                                          headers=self.headers,
                                                          timeout=(30, 60),
                                                          verify=False)
            if BoolResult:
                self.para(r.text, row, url)
                sql = "update journal set stat=1 where url='{}'".format(row[1])
                self.mysqlutile.ExeSqlToDB(sql)
            else:
                print("下载失败")

    def para(self, html, row, url):
        selecter = parsel.Selector(text=html)
        ullisttag = selecter.xpath('//*[@id="mainContent"]/div[2]/div[1]/ul')
        Listpara = []
        for ultag in ullisttag:
            lilisttag = ultag.xpath("./li")
            for litag in lilisttag:
                href = litag.xpath('./a/@href').get()
                title = litag.xpath('./a/text()').get()
                sourcename = row[0]
                journal = row[2]
                Listpara.append((href, title, sourcename, journal, url))
        sql = "replace into volissue(`url`,`title`,`sourcename`,`journal`,`jurl`) values (%s,%s,%s,%s,%s)"
        self.mysqlutile.ExeSqlMany(sql, Listpara)


def main():
    down = DownJournal()
    down.select()


if __name__ == "__main__":
    main()
